package com.spider.silence.downloader;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.spider.silence.abstarct.downloader.AbstractLoginDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.SinaWeiBoTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.*;
import org.apache.log4j.Logger;
import org.dom4j.Element;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.support.ui.ExpectedCondition;
import org.openqa.selenium.support.ui.WebDriverWait;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by 夏茂轩 on 2017/1/3.
 *
 * @description 新浪微博下载器
 */
public class SinaWeiBoDownLoader extends AbstractLoginDownLoader {
    private Logger logger = Logger.getLogger(SinaWeiBoDownLoader.class);
    private String userName;
    private String pwd;
    private Map<String,File> files=null;
    public Map<String,File> getFiles(){
        return files;
    }
    public SinaWeiBoDownLoader(String userName, String pwd) {
        this.userName = userName;
        this.pwd = pwd;
    }

    public SinaWeiBoDownLoader() {
    }

    @Override
    protected HashSet<Cookie> login() {
        ChromeDriver driver = null;
        try {
            driver = ChromeDriverUtils.getDriver();
            if (userName == null) {
                driver.get("http://weibo.com/");
                try {
                    Thread.sleep((long) (2000 + Math.random() * 2000));
                } catch (InterruptedException e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                }
            } else {
                driver.get("https://passport.weibo.cn/signin/login");
                Thread.sleep((long) (2000 + Math.random() * 2000));
                WebDriverWait wait = new WebDriverWait(driver, 10);
                wait.until(new ExpectedCondition<WebElement>() {
                    @Override
                    public WebElement apply(WebDriver d) {
                        return d.findElement(By.cssSelector("#loginName"));
                    }
                }).sendKeys(userName);
                wait.until(new ExpectedCondition<WebElement>() {
                    @Override
                    public WebElement apply(WebDriver d) {
                        return d.findElement(By.cssSelector("#loginPassword"));
                    }
                }).sendKeys(pwd);
                Thread.sleep((long) (500 + Math.random()));
                wait.until(new ExpectedCondition<WebElement>() {
                    @Override
                    public WebElement apply(WebDriver d) {
                        return d.findElement(By.cssSelector("#loginAction"));
                    }
                }).click();
                Thread.sleep((long) (2000 + Math.random()));
            }
            return (HashSet<Cookie>) driver.manage().getCookies();
        } catch (InterruptedException e) {
            logger.error(e.getMessage(),e);
            e.printStackTrace();
        } finally {
            driver.quit();
        }
        return null;
    }

    public void  test(String url) throws Exception {
        Page page =getHtml(url);
        String script=page.getHtml().getDocument().select("script").get(1).html();
        script="var location={href:''};"+script;
        script+=";function test(){" +
                "return config.pageConfig.value;" +
                "}; test();";
        ScriptEngineManager manager = new ScriptEngineManager();
        ScriptEngine engine = manager.getEngineByName( "JavaScript" );
        System.out.println( "Result:" + engine.eval( script ) );
    }
    public void generateTask() {
        org.dom4j.Document document1 = XmlUtil.getConf("META-INF/SinaWeibo.xml");
        List<Element> elementssAll=document1.getRootElement().element("condition-list").elements();
        Random rand =new Random();
        int needIndex=rand.nextInt(elementssAll.size()-20);
        //Collections.shuffle(elementssAll);
        List<Element> elementss=elementssAll.subList(needIndex,needIndex+20);
        for(int i=0;i<elementss.size();i++){
            try {
                Element element=elementss.get(i);
                String condition=element.element("value").getText();
                String type=element.element("type").getText();
                if("1".equals(type)){
                    //关键词搜索
                    String moduleUrl="http://m.weibo.cn/container/getIndex?containerid=100103type%3D2%26q%3D"+condition;

                    boolean flag = true;
                    long startTime = System.currentTimeMillis();
                    long nowTime = startTime;
                    while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("Sina.TimeOut"))) {
                        try {
                            Document document = null;
                            Page pageContent = null;
                            String getDataUrl="";
                            getDataUrl=moduleUrl;
                            pageContent = this.getHtml(moduleUrl, null);
                            if (null == pageContent) {

                            } else if (pageContent.getStatusCode() == 0) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                            } else if (pageContent.getStatusCode() == 200) {
                                flag = false;
                                String moduleJson=pageContent.getRawText();
                                if(null!=moduleJson&&!"".equals(moduleJson)){
                                    JSONObject moduleJsonObj = JSONObject.parseObject(moduleJson);
                                    if(null!=moduleJsonObj.getString("ok")&&"1".equals(moduleJsonObj.getString("ok"))){
                                        JSONArray cards=moduleJsonObj.getJSONArray("cards");
                                        JSONObject cardss=cards.getJSONObject(0);
                                        JSONArray cardGroup=cardss.getJSONArray("card_group");
                                        for(int j=0;j<cardGroup.size();j++){
                                            String itemId=cardGroup.getJSONObject(j).getJSONObject("mblog").getString("id");
                                            if(null!=itemId&&!"".equals(itemId)){
                                                SinaWeiBoTask task=new SinaWeiBoTask(itemId);
                                                CommonTaskManager.add("sinaWeiBo", task);
                                            }
                                        }
                                    }
                                }

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                            } else {

                            }

                        } catch (Exception e1) {
                            //System.out.println("异常信息："+e1.getMessage());
                            logger.error(e1.getMessage(), e1);
                        }finally{
                            nowTime = System.currentTimeMillis();
                        }
                    }

                }else if("2".equals(type)) {
                    //指定用户

                    String moduleUrl1 = "http://m.weibo.cn/container/getIndex?type=uid&value="+condition;

                    boolean flag1 = true;
                    long startTime1 = System.currentTimeMillis();
                    long nowTime1 = startTime1;
                    while (flag1 && nowTime1 - startTime1 < Long.valueOf(PropertiesUtil.getVal("Sina.TimeOut"))) {
                        try {
                            Document document11 = null;
                            Page pageContent1 = null;
                            String getDataUrl1="";
                            getDataUrl1=moduleUrl1;
                            pageContent1 = this.getHtml(moduleUrl1, null);
                            if (null == pageContent1) {

                            } else if (pageContent1.getStatusCode() == 0) {

                            } else if (String.valueOf(pageContent1.getStatusCode()).startsWith("1")) {

                            } else if (pageContent1.getStatusCode() == 200) {
                                flag1 = false;
                                String moduleJson1=pageContent1.getRawText();
                                if (null != moduleJson1 && !"".equals(moduleJson1)) {
                                    JSONObject moduleJsonObj1 = JSONObject.parseObject(moduleJson1);
                                    if (null != moduleJsonObj1.getString("ok") && "1".equals(moduleJsonObj1.getString("ok"))) {
                                        JSONArray tabsArr = moduleJsonObj1.getJSONObject("tabsInfo").getJSONArray("tabs");
                                         if(null!=tabsArr&&tabsArr.size()>0) {
                                            String containerid = tabsArr.getJSONObject(1).getString("containerid");
                                            if (null != containerid && !"".equals(containerid)) {
                                                String moduleUrl = "http://m.weibo.cn/container/getIndex?containerid="+containerid+"&value=" + condition;

                                                boolean flag = true;
                                                long startTime = System.currentTimeMillis();
                                                long nowTime = startTime;
                                                while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("Sina.TimeOut"))) {
                                                    try {
                                                        Document document = null;
                                                        Page pageContent = null;
                                                        String getDataUrl="";
                                                        getDataUrl=moduleUrl;
                                                        pageContent = this.getHtml(moduleUrl, null);
                                                        if (null == pageContent) {

                                                        } else if (pageContent.getStatusCode() == 0) {

                                                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                                                        } else if (pageContent.getStatusCode() == 200) {
                                                            flag = false;
                                                            String moduleJson=pageContent.getRawText();
                                                            if (null != moduleJson && !"".equals(moduleJson)) {
                                                                JSONObject moduleJsonObj = JSONObject.parseObject(moduleJson);
                                                                if (null != moduleJsonObj.getString("ok") && "1".equals(moduleJsonObj.getString("ok"))) {
                                                                    JSONArray cards = moduleJsonObj.getJSONArray("cards");
                                                                    for (int j = 0; j < cards.size(); j++) {
                                                                        if(null!=cards.getJSONObject(j).getJSONObject("mblog")&&!"".equals(cards.getJSONObject(j).getJSONObject("mblog"))){
                                                                            String itemId = cards.getJSONObject(j).getJSONObject("mblog").getString("id");
                                                                            if (null != itemId && !"".equals(itemId)) {
                                                                                SinaWeiBoTask task = new SinaWeiBoTask(itemId);
                                                                                CommonTaskManager.add("sinaWeiBo", task);

                                                                            }
                                                                        }
                                                                    }
                                                                }

                                                            }

                                                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                                                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                                                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                                                        } else {

                                                        }

                                                    } catch (Exception e1) {
                                                        //System.out.println("异常信息："+e1.getMessage());
                                                        logger.error(e1.getMessage(), e1);
                                                    }finally{
                                                        nowTime = System.currentTimeMillis();
                                                    }
                                                }

                                            }
                                        }
                                    }

                                }

                            } else if (String.valueOf(pageContent1.getStatusCode()).startsWith("3")) {

                            } else if (String.valueOf(pageContent1.getStatusCode()).startsWith("4")) {

                            } else if (String.valueOf(pageContent1.getStatusCode()).startsWith("5")) {

                            } else {

                            }

                        } catch (Exception e1) {
                            //System.out.println("异常信息："+e1.getMessage());
                            logger.error(e1.getMessage(), e1);
                        }finally{
                            nowTime1 = System.currentTimeMillis();
                        }
                    }

                }
            } catch (Exception e) {
                continue;
            }
        }

    }

    public JSONObject crawlItem(String itemId) {
        files=new ConcurrentHashMap<>();
        String itemUrl = "http://m.weibo.cn/status/%s";
        JSONObject o = new JSONObject();
        JSONObject data = new JSONObject();
        try {
            String document = this.getString(String.format(itemUrl, new String[]{itemId}),null);
            Pattern pattern = Pattern.compile("render_data = \\[([\\w\\W]*?)\\]\\[0\\]\\s\\|\\|\\s\\{\\}");
            Matcher matcher = pattern.matcher(document);
            String json = "{}";
            while (matcher.find()) {
                json = matcher.group(1);
            }
            JSONObject status = JSONObject.parseObject(json);
            JSONObject item = status.getJSONObject("status");
            String publish_time = DateUtils.formatDateTime(new Date(item.getString("created_at")));
            data.put("publish_time", publish_time);
            Date now = new Date();
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            data.put("spider_time", sdf.format(now));
            String content = item.getString("text");
            data.put("content", FileUtils.filterImg(content));
            JSONArray img_url=new JSONArray();
            List<String> imgList=FileUtils.filterImgSrc(content);
            if(imgList!=null && imgList.size()>0){
                for(String img : imgList){
                    img_url.add(img);
                    System.out.println(img);
                }
            }
            data.put("img_urls",img_url);//发布的图片
            long reproduce = item.getLong("reposts_count");//转发数
            data.put("reproduce_count", reproduce);
            long reply_count = item.getLong("comments_count");//回复数
            data.put("reply_count", reply_count);
            long like = item.getLong("attitudes_count");//点赞数
            data.put("like_count", like);
            if (item.containsKey("reads_count")) {
                long read_count = item.getLong("reads_count");//阅读数
                data.put("read_count", read_count);
            }
            String url = String.format(itemUrl, new String[]{itemId});
            data.put("url",url);
            String id = itemId;
            data.put("id",id);
            JSONObject user = item.getJSONObject("user");
            long publish_user_id = user.getLong("id");
            data.put("publish_user_id", publish_user_id);
            String publish_user = user.getString("screen_name");
            data.put("publish_user", publish_user);
            String publish_user_photo = user.getString("profile_image_url");
            data.put("publish_user_photo", publish_user_photo);

            long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
            String filePrefix="SinaWeiBo";//文件前缀
            String fileName=filePrefix+"_"+tsp+"_"+publish_user_id+"_"+id;
            String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
            String speechesId="SinaWeiBo_speeches_"+publishDate+"_"+fileName;
            File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document);
            files.put(speechesId+"_pageNo_1",file);
            //帖子首页的文件


            if (item.containsKey("pics")) {
                JSONArray img_urls = new JSONArray();//图片
                JSONArray org_img_array = item.getJSONArray("pics");
                for (int i = 0; i < org_img_array.size(); i++) {
                    JSONObject img_obj = org_img_array.getJSONObject(i);
                    if (img_obj.containsKey("large")) {
                        img_urls.add(img_obj.getJSONObject("large").getString("url"));
                    } else {
                        img_urls.add(img_obj.getString("url"));
                    }
                }
                data.put("img_urls", img_urls);
            }

            if(reply_count>0){
                String replyUrl = "http://m.weibo.cn/api/comments/show?id=%s";

                boolean flag = true;
                long startTime = System.currentTimeMillis();
                long nowTime = startTime;
                while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("Sina.TimeOut"))) {
                    try {
                        Page pageContent = null;
                        pageContent = this.getHtml(String.format(replyUrl, new String[]{itemId}) + "&page=1", null);
                        if (null == pageContent) {

                        } else if (pageContent.getStatusCode() == 0) {

                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                        } else if (pageContent.getStatusCode() == 200) {
                            try {
                                Document replyDoc=pageContent.getHtml().getDocument();
                                ResultItems items= pageContent.getResultItems();

                                if(null!=replyDoc&&null!=replyDoc.text()){
                                    JSONObject firstReplyPage = JSONObject.parseObject(replyDoc.text());
                                    int page = firstReplyPage.getInteger("max");
                                    JSONArray reply_nodes = new JSONArray();
                                    if (firstReplyPage.containsKey("ok")) {
                                        if (firstReplyPage.getInteger("ok") == 1) {
                                            if (firstReplyPage.containsKey("data")) {
                                                JSONArray replyArray = firstReplyPage.getJSONArray("data");
                                                for (int i = 0; i < replyArray.size(); i++) {
                                                    try {
                                                        JSONObject reply_result = new JSONObject();
                                                        JSONObject reply = replyArray.getJSONObject(i);
                                                        String reply_publish_time = DateUtils.formatSinaDateTime(reply.getString("created_at"));
                                                        reply_result.put("publish_time", reply_publish_time);
                                                        String reply_content = reply.getString("text");
                                                        reply_result.put("content", FileUtils.filterImg(reply_content));
                                                        JSONArray reply_img_url=new JSONArray();
                                                        List<String> replyImgList=FileUtils.filterImgSrc(reply_content);
                                                        if(replyImgList!=null && replyImgList.size()>0){
                                                            for(String img : replyImgList){
                                                                reply_img_url.add(img);
                                                                System.out.println("回复图片"+img);
                                                            }
                                                        }
                                                        reply_result.put("img_urls",reply_img_url);//发布的图片
                                                        if(reply.containsKey("like_counts")){
                                                            long reply_likes = 0;
                                                            if(reply.getString("like_counts")==null||reply.getString("like_counts").isEmpty()){
                                                                reply_likes=0;
                                                            }else{
                                                                reply_likes=reply.getLong("like_counts");
                                                            }
                                                            reply_result.put("like_count", reply_likes);
                                                        }
                                                        JSONObject reply_user = reply.getJSONObject("user");
                                                        int reply_user_id = reply_user.getInteger("id");
                                                        reply_result.put("publish_user_id", reply_user_id);
                                                        String reply_user_name = reply_user.getString("screen_name");
                                                        reply_result.put("publish_user", reply_user_name);
                                                        String replyText=reply.getString("reply_text");
                                                        if(replyText!=null && !"".equals(replyText)){
                                                            reply_result.put("reply_nodes",getReplysReply(reply.getString("text")));
                                                        }
                                                        String reply_user_photo = reply_user.getString("profile_image_url");
                                                        reply_result.put("publish_user_photo", reply_user_photo);
                                                        reply_result.put("parent_id", id);
                                                        reply_result.put("ancestor_id", id);
                                                        reply_nodes.add(reply_result);
                                                    } catch (Exception e) {
                                                        logger.error(e.getMessage(),e);
                                                        continue;
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    //只爬取第一页
                                    List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                                    if (page <= -2) { //添加子回复
                                        int threadNum = Integer.valueOf(PropertiesUtil.getVal("Sina.ReplyThreadCount"));
                                        List<FutureTask<JSONArray>> futureTasks = Lists.newArrayList();
                                        ExecutorService executorService = Executors.newFixedThreadPool(threadNum);

                                        try {
                                            ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                            for (int i = 0; i < splitPages.size(); i++) {
                                                SinaWeiBoDownLoader downLoader = this.cloneDownLoader();
                                                ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), String.format(replyUrl, new String[]{itemId}), id,speechesId);
                                                FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                                futureTasks.add(futureTask);
                                                executorService.submit(futureTask);
                                            }
                                            for (FutureTask<JSONArray> futureTask : futureTasks) {
                                                JSONArray replyArrays = futureTask.get();
                                                for (int i = 0; i < replyArrays.size(); i++) {
                                                    JSONObject object = replyArrays.getJSONObject(i);
                                                    reply_nodes.add(object);
                                                }
                                            }
                                            if (reply_nodes.size() > 0 && reply_nodes != null) {
                                                for (int i = 0; i < reply_nodes.size(); i++) {
                                                    JSONObject reply_object = reply_nodes.getJSONObject(i);
                                                    Map<String, Object> map = reply_object;
                                                    list.add(map);
                                                }
                                            }
                                            Collections.sort(list, new Comparator<Map<String, Object>>() {

                                                public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                                    String s1 = o1.get("publish_time").toString();
                                                    String s2 = o2.get("publish_time").toString();
                                                    int flag = s1.compareTo(s2);
                                                    return flag;
                                                }
                                            });

                                        } catch (Exception e1) {
                                            logger.error(e1.getMessage(),e1);
                                        } finally {
                                            executorService.shutdown();
                                        }

                                    }else{
                                        if (reply_nodes.size() > 0 && reply_nodes != null) {
                                            for (int i = 0; i < reply_nodes.size(); i++) {
                                                JSONObject reply_object = reply_nodes.getJSONObject(i);
                                                Map<String, Object> map = reply_object;
                                                list.add(map);
                                            }
                                        }
                                    }
                                    data.put("reply_nodes", list);
                                }

                            } catch (Exception e) {
                                logger.error(e.getMessage(),e);
                            }finally{
                                flag = false;
                            }

                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                        } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                        } else {

                        }

                    } catch (Exception e1) {
                        logger.error(e1.getMessage(),e1);
                    }finally{
                        if(flag==true){
                            if(data.containsKey("reply_nodes")){
                                data.remove("reply_nodes");
                            }
                        }
                        nowTime = System.currentTimeMillis();
                    }
                }
            }

        } catch (Exception e) {
            o = new JSONObject();
            data = new JSONObject();
            logger.error(e.getMessage(),e);
        }
        o.put("data", data);
        return o;
    }

    /**
     * 爬取回复的回复
     * @param reply
     * @return
     */
    private JSONObject getReplysReply(String reply){
        JSONObject object=new JSONObject();
        if(reply.contains("</a>")) {
            String username = reply.split("</a>")[0].split("@")[1];
            String url = "http://m.weibo.cn/n/" + username;
            object.put("publish_user", username); //回复用户名
            String[] replyContent = reply.split(":");
            String content = "";
            if (replyContent.length > 2) {
                for (int i = 1; i < replyContent.length; i++) {
                    content += replyContent[i] + ":";
                }
                content.substring(0, content.length() - 1);
            } else {
                content = replyContent[1];
            }
            object.put("content", content);
            Page pageContent = null;
            try {
                pageContent = getHtml(url);
                String script=pageContent.getHtml().getDocument().select("script").get(1).html();
                script="var location={href:''};"+script;
                script+=";function test(){" +
                        "return config.pageConfig.value;" +
                        "}; test();";
                ScriptEngineManager manager = new ScriptEngineManager();
                ScriptEngine engine = manager.getEngineByName( "JavaScript" );
                object.put("publish_user_id",  engine.eval( script ));
            } catch (Exception e) {
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
        }
        System.out.println(object);
        return object;
    }

    /**
     * 回复下载器
     */
    class ReplyCrawler implements Callable<JSONArray> {
        private SinaWeiBoDownLoader loader;
        private ArrayList<Integer> pages;
        private String url;
        private String parentId;
        private String speeches_id;

        @Override
        public JSONArray call() throws Exception {
            JSONArray array = new JSONArray();
            for (int m = 0; m < pages.size(); m++) {
                String getUrl = url + "&page=" + pages.get(m);
                String document = loader.getString(getUrl,null);
                File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(m),document);
                files.put(speeches_id+"_pageNo_"+pages.get(m),file);
                try {
                    JSONObject rto = JSONObject.parseObject(document);
                    if (rto.containsKey("ok")) {
                        if (rto.getInteger("ok") == 1) {
                            if (rto.containsKey("data")) {
                                JSONArray replyArray = rto.getJSONArray("data");
                                //System.out.println(replyArray.size());
                                for (int i = 0; i < replyArray.size(); i++) {
                                    JSONObject reply_result = new JSONObject();
                                    JSONObject reply = replyArray.getJSONObject(i);
                                    try {
                                        String reply_publish_time = DateUtils.formatSinaDateTime(reply.getString("created_at"));
                                        reply_result.put("publish_time", reply_publish_time);
                                        String reply_content = reply.getString("text");
                                        reply_result.put("content", FileUtils.filterImg(reply_content));

                                        JSONArray reply_img_url=new JSONArray();
                                        List<String> replyImgList=FileUtils.filterImgSrc(reply_content);
                                        if(replyImgList!=null && replyImgList.size()>0){
                                            for(String img : replyImgList){
                                                reply_img_url.add(img);
                                                System.out.println("回复图片"+img);
                                            }
                                        }
                                        reply_result.put("img_urls",reply_img_url);//发布的图片

                                        if (reply.containsKey("like_counts")) {
                                            long reply_likes = 0;
                                            if(reply.getString("like_counts")==null||reply.getString("like_counts").isEmpty()){
                                                reply_likes=0;
                                            }else{
                                                reply_likes=reply.getLong("like_counts");
                                            }
                                            reply_result.put("like_count", reply_likes);
                                        }
                                        JSONObject reply_user = reply.getJSONObject("user");
                                        int reply_user_id = reply_user.getInteger("id");
                                        reply_result.put("publish_user_id", reply_user_id);
                                        String reply_user_name = reply_user.getString("screen_name");
                                        reply_result.put("publish_user", reply_user_name);
                                        String reply_user_photo = reply_user.getString("profile_image_url");
                                        reply_result.put("publish_user_photo", reply_user_photo);
                                        reply_result.put("parent_id", parentId);
                                        reply_result.put("ancestor_id", parentId);
                                        array.add(reply_result);
                                    } catch (Exception e) {
                                        logger.error(e.getMessage(),e);
                                        e.printStackTrace();
                                        continue;
                                    }
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error(e.getMessage(),e);
                    //System.out.println(getUrl);
                    continue;
                }
            }
            return array;
        }

        public ReplyCrawler(SinaWeiBoDownLoader downLoader, ArrayList<Integer> pages, String url, String parentId,String speeches_id) {
            this.loader = downLoader;
            this.pages = pages;
            this.url = url;
            this.parentId = parentId;
            this.speeches_id=speeches_id;

        }
    }

    public SinaWeiBoDownLoader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        SinaWeiBoDownLoader loader = new SinaWeiBoDownLoader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }
}
